{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install stanfordcorenlp\n",
    "from stanfordcorenlp import StanfordCoreNLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 基本使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenize: ['Guangdong', 'University', 'of', 'Foreign', 'Studies', 'is', 'located', 'in', 'Guangzhou', '.']\n",
      "Part of Speech: [('Guangdong', 'NNP'), ('University', 'NNP'), ('of', 'IN'), ('Foreign', 'NNP'), ('Studies', 'NNPS'), ('is', 'VBZ'), ('located', 'JJ'), ('in', 'IN'), ('Guangzhou', 'NNP'), ('.', '.')]\n",
      "Named Entities: [('Guangdong', 'ORGANIZATION'), ('University', 'ORGANIZATION'), ('of', 'ORGANIZATION'), ('Foreign', 'ORGANIZATION'), ('Studies', 'ORGANIZATION'), ('is', 'O'), ('located', 'O'), ('in', 'O'), ('Guangzhou', 'CITY'), ('.', 'O')]\n",
      "Constituency Parsing: (ROOT\n",
      "  (S\n",
      "    (NP\n",
      "      (NP (NNP Guangdong) (NNP University))\n",
      "      (PP (IN of)\n",
      "        (NP (NNP Foreign) (NNPS Studies))))\n",
      "    (VP (VBZ is)\n",
      "      (ADJP (JJ located)\n",
      "        (PP (IN in)\n",
      "          (NP (NNP Guangzhou)))))\n",
      "    (. .)))\n",
      "Dependency Parsing: [('ROOT', 0, 7), ('compound', 2, 1), ('nsubjpass', 7, 2), ('case', 5, 3), ('compound', 5, 4), ('nmod', 2, 5), ('auxpass', 7, 6), ('case', 9, 8), ('nmod', 7, 9), ('punct', 7, 10)]\n"
     ]
    }
   ],
   "source": [
    "from stanfordcorenlp import StanfordCoreNLP\n",
    "\n",
    "nlp = StanfordCoreNLP(r'/home/jasonhaven/Downloads/stanford-corenlp-full-2018-10-05')\n",
    "sentence = 'Guangdong University of Foreign Studies is located in Guangzhou.'\n",
    "print('Tokenize:', nlp.word_tokenize(sentence))\n",
    "print('Part of Speech:', nlp.pos_tag(sentence))\n",
    "print('Named Entities:', nlp.ner(sentence))\n",
    "print('Constituency Parsing:', nlp.parse(sentence))\n",
    "print('Dependency Parsing:', nlp.dependency_parse(sentence))\n",
    "\n",
    "nlp.close() # Do not forget to close! The backend server will consume a lot memery."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['清华', '大学', '位于', '北京', '。']\n",
      "[('清华', 'NR'), ('大学', 'NN'), ('位于', 'VV'), ('北京', 'NR'), ('。', 'PU')]\n",
      "[('清华', 'ORGANIZATION'), ('大学', 'ORGANIZATION'), ('位于', 'O'), ('北京', 'STATE_OR_PROVINCE'), ('。', 'O')]\n",
      "(ROOT\n",
      "  (IP\n",
      "    (NP (NR 清华) (NN 大学))\n",
      "    (VP (VV 位于)\n",
      "      (NP (NR 北京)))\n",
      "    (PU 。)))\n",
      "[('ROOT', 0, 3), ('compound:nn', 2, 1), ('nsubj', 3, 2), ('dobj', 3, 4), ('punct', 3, 5)]\n"
     ]
    }
   ],
   "source": [
    "# 中文\n",
    "sentence = '清华大学位于北京。'\n",
    "with StanfordCoreNLP(r'/home/jasonhaven/Downloads/stanford-corenlp-full-2018-10-05',lang='zh') as nlp:\n",
    "    print(nlp.word_tokenize(sentence))\n",
    "    print(nlp.pos_tag(sentence))\n",
    "    print(nlp.ner(sentence))\n",
    "    print(nlp.parse(sentence))\n",
    "    print(nlp.dependency_parse(sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n",
      "<?xml-stylesheet href=\"CoreNLP-to-HTML.xsl\" type=\"text/xsl\"?>\r\n",
      "<root>\r\n",
      "  <document>\r\n",
      "    <sentences>\r\n",
      "      <sentence id=\"1\">\r\n",
      "        <tokens>\r\n",
      "          <token id=\"1\">\r\n",
      "            <word>Guangdong</word>\r\n",
      "            <CharacterOffsetBegin>0</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>9</CharacterOffsetEnd>\r\n",
      "            <POS>NNP</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"2\">\r\n",
      "            <word>University</word>\r\n",
      "            <CharacterOffsetBegin>10</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>20</CharacterOffsetEnd>\r\n",
      "            <POS>NNP</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"3\">\r\n",
      "            <word>of</word>\r\n",
      "            <CharacterOffsetBegin>21</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>23</CharacterOffsetEnd>\r\n",
      "            <POS>IN</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"4\">\r\n",
      "            <word>Foreign</word>\r\n",
      "            <CharacterOffsetBegin>24</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>31</CharacterOffsetEnd>\r\n",
      "            <POS>NNP</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"5\">\r\n",
      "            <word>Studies</word>\r\n",
      "            <CharacterOffsetBegin>32</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>39</CharacterOffsetEnd>\r\n",
      "            <POS>NNPS</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"6\">\r\n",
      "            <word>is</word>\r\n",
      "            <CharacterOffsetBegin>40</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>42</CharacterOffsetEnd>\r\n",
      "            <POS>VBZ</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"7\">\r\n",
      "            <word>located</word>\r\n",
      "            <CharacterOffsetBegin>43</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>50</CharacterOffsetEnd>\r\n",
      "            <POS>JJ</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"8\">\r\n",
      "            <word>in</word>\r\n",
      "            <CharacterOffsetBegin>51</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>53</CharacterOffsetEnd>\r\n",
      "            <POS>IN</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"9\">\r\n",
      "            <word>Guangzhou</word>\r\n",
      "            <CharacterOffsetBegin>54</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>63</CharacterOffsetEnd>\r\n",
      "            <POS>NNP</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"10\">\r\n",
      "            <word>.</word>\r\n",
      "            <CharacterOffsetBegin>63</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>64</CharacterOffsetEnd>\r\n",
      "            <POS>.</POS>\r\n",
      "          </token>\r\n",
      "        </tokens>\r\n",
      "      </sentence>\r\n",
      "      <sentence id=\"2\">\r\n",
      "        <tokens>\r\n",
      "          <token id=\"1\">\r\n",
      "            <word>GDUFS</word>\r\n",
      "            <CharacterOffsetBegin>65</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>70</CharacterOffsetEnd>\r\n",
      "            <POS>NN</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"2\">\r\n",
      "            <word>is</word>\r\n",
      "            <CharacterOffsetBegin>71</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>73</CharacterOffsetEnd>\r\n",
      "            <POS>VBZ</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"3\">\r\n",
      "            <word>active</word>\r\n",
      "            <CharacterOffsetBegin>74</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>80</CharacterOffsetEnd>\r\n",
      "            <POS>JJ</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"4\">\r\n",
      "            <word>in</word>\r\n",
      "            <CharacterOffsetBegin>81</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>83</CharacterOffsetEnd>\r\n",
      "            <POS>IN</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"5\">\r\n",
      "            <word>a</word>\r\n",
      "            <CharacterOffsetBegin>84</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>85</CharacterOffsetEnd>\r\n",
      "            <POS>DT</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"6\">\r\n",
      "            <word>full</word>\r\n",
      "            <CharacterOffsetBegin>86</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>90</CharacterOffsetEnd>\r\n",
      "            <POS>JJ</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"7\">\r\n",
      "            <word>range</word>\r\n",
      "            <CharacterOffsetBegin>91</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>96</CharacterOffsetEnd>\r\n",
      "            <POS>NN</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"8\">\r\n",
      "            <word>of</word>\r\n",
      "            <CharacterOffsetBegin>97</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>99</CharacterOffsetEnd>\r\n",
      "            <POS>IN</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"9\">\r\n",
      "            <word>international</word>\r\n",
      "            <CharacterOffsetBegin>100</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>113</CharacterOffsetEnd>\r\n",
      "            <POS>JJ</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"10\">\r\n",
      "            <word>cooperation</word>\r\n",
      "            <CharacterOffsetBegin>114</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>125</CharacterOffsetEnd>\r\n",
      "            <POS>NN</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"11\">\r\n",
      "            <word>and</word>\r\n",
      "            <CharacterOffsetBegin>126</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>129</CharacterOffsetEnd>\r\n",
      "            <POS>CC</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"12\">\r\n",
      "            <word>exchanges</word>\r\n",
      "            <CharacterOffsetBegin>130</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>139</CharacterOffsetEnd>\r\n",
      "            <POS>NNS</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"13\">\r\n",
      "            <word>in</word>\r\n",
      "            <CharacterOffsetBegin>140</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>142</CharacterOffsetEnd>\r\n",
      "            <POS>IN</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"14\">\r\n",
      "            <word>education</word>\r\n",
      "            <CharacterOffsetBegin>143</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>152</CharacterOffsetEnd>\r\n",
      "            <POS>NN</POS>\r\n",
      "          </token>\r\n",
      "          <token id=\"15\">\r\n",
      "            <word>.</word>\r\n",
      "            <CharacterOffsetBegin>152</CharacterOffsetBegin>\r\n",
      "            <CharacterOffsetEnd>153</CharacterOffsetEnd>\r\n",
      "            <POS>.</POS>\r\n",
      "          </token>\r\n",
      "        </tokens>\r\n",
      "      </sentence>\r\n",
      "    </sentences>\r\n",
      "  </document>\r\n",
      "</root>\r\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Use an existing server\n",
    "nlp = StanfordCoreNLP('http://corenlp.run', port=80)\n",
    "text = 'Guangdong University of Foreign Studies is located in Guangzhou. ' \\\n",
    "       'GDUFS is active in a full range of international cooperation and exchanges in education. '\n",
    "\n",
    "props={'annotators': 'tokenize,ssplit,pos','pipelineLanguage':'en','outputFormat':'xml'}\n",
    "print(nlp.annotate(text, properties=props))\n",
    "nlp.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Use device: cpu\n",
      "---\n",
      "Loading: tokenize\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_tokenizer.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "---\n",
      "Loading: pos\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_tagger.pt', 'pretrain_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt.pretrain.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "---\n",
      "Loading: lemma\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_lemmatizer.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "Building an attentional Seq2Seq model...\n",
      "Using a Bi-LSTM encoder\n",
      "Using soft attention for LSTM.\n",
      "Finetune all embeddings.\n",
      "[Running seq2seq lemmatizer with edit classifier]\n",
      "---\n",
      "Loading: depparse\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_parser.pt', 'pretrain_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt.pretrain.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "Done loading processors!\n",
      "---\n",
      "('Barack', '4', 'nsubj:pass')\n",
      "('Obama', '1', 'flat')\n",
      "('was', '4', 'aux:pass')\n",
      "('born', '0', 'root')\n",
      "('in', '6', 'case')\n",
      "('Hawaii', '4', 'obl')\n",
      "('.', '4', 'punct')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pytorch/aten/src/ATen/native/LegacyDefinitions.cpp:14: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n"
     ]
    }
   ],
   "source": [
    "import stanfordnlp\n",
    "#stanfordnlp.download('en')\n",
    "nlp = stanfordnlp.Pipeline() \n",
    "doc = nlp(\"Barack Obama was born in Hawaii.  He was elected president in 2008.\")\n",
    "doc.sentences[0].print_dependencies()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 下载中文模型失败，可能是网络问题\n",
    "stanfordnlp.download('zh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Use device: cpu\n",
      "---\n",
      "Loading: tokenize\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_tokenizer.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "---\n",
      "Loading: pos\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_tagger.pt', 'pretrain_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt.pretrain.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "---\n",
      "Loading: lemma\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_lemmatizer.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "Building an attentional Seq2Seq model...\n",
      "Using a Bi-LSTM encoder\n",
      "Using soft attention for LSTM.\n",
      "Finetune all embeddings.\n",
      "[Running seq2seq lemmatizer with edit classifier]\n",
      "---\n",
      "Loading: depparse\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_parser.pt', 'pretrain_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt.pretrain.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "Done loading processors!\n",
      "---\n",
      "---\n",
      "Building pipeline...\n",
      "Use device: cpu\n",
      "---\n",
      "Loading: tokenize\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_tokenizer.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "---\n",
      "Loading: pos\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_tagger.pt', 'pretrain_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt.pretrain.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "---\n",
      "Loading: lemma\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_lemmatizer.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "Building an attentional Seq2Seq model...\n",
      "Using a Bi-LSTM encoder\n",
      "Using soft attention for LSTM.\n",
      "Finetune all embeddings.\n",
      "[Running seq2seq lemmatizer with edit classifier]\n",
      "---\n",
      "Loading: depparse\n",
      "With settings: \n",
      "{'model_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt_parser.pt', 'pretrain_path': '/home/jasonhaven/stanfordnlp_resources/en_ewt_models/en_ewt.pretrain.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
      "Done loading processors!\n",
      "---\n",
      "\n",
      "Input: 達沃斯世界經濟論壇是每年全球政商界領袖聚在一起的年度盛事。\n",
      "The tokenizer split the input into 1 sentences.\n",
      "---\n",
      "tokens of first sentence: \n",
      "<Token index=1;words=[<Word index=1;text=達沃斯世界經濟論壇是每年全球政商界領袖聚在一起的年度盛事。;lemma=達沃斯世界經濟論壇是每年全球政商界領袖聚在一起的年度盛事。;upos=NOUN;xpos=NN;feats=Number=Sing;governor=0;dependency_relation=root>]>\n",
      "\n",
      "---\n",
      "dependency parse of first sentence: \n",
      "('達沃斯世界經濟論壇是每年全球政商界領袖聚在一起的年度盛事。', '0', 'root')\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pytorch/aten/src/ATen/native/LegacyDefinitions.cpp:14: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n"
     ]
    }
   ],
   "source": [
    "nlp = stanfordnlp.Pipeline()\n",
    "lang = 'zh'\n",
    "sentence = '達沃斯世界經濟論壇是每年全球政商界領袖聚在一起的年度盛事。'\n",
    "\n",
    "# set up a pipeline\n",
    "print('---')\n",
    "print('Building pipeline...')\n",
    "pipeline = stanfordnlp.Pipeline()\n",
    "# process the document\n",
    "doc = pipeline(sentence)\n",
    "# access nlp annotations\n",
    "print('')\n",
    "print('Input: {}'.format(sentence))\n",
    "print(\"The tokenizer split the input into {} sentences.\".format(len(doc.sentences)))\n",
    "print('---')\n",
    "print('tokens of first sentence: ')\n",
    "doc.sentences[0].print_tokens()\n",
    "print('')\n",
    "print('---')\n",
    "print('dependency parse of first sentence: ')\n",
    "doc.sentences[0].print_dependencies()\n",
    "print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
